{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HER2 One Scanner - Aperio FDA\n",
    "\n",
    "- 5-Fold (80/20) split, No Holdout Set\n",
    "- Truth = Categorical from Mean of 7 continuous scores \n",
    "- Epoch at automatic Stop when loss<.001 change \n",
    "- LeNet model, 10 layers, Dropout (0.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/usr/lib/python2.7/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "from keras.callbacks import EarlyStopping\n",
    "from PIL import Image\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D, Lambda\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import roc_curve, auc, classification_report\n",
    "import csv\n",
    "import cv2\n",
    "import scipy\n",
    "import os\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#For single scanner\n",
    "BASE_PATH = '/home/diam/Desktop/HER2_data_augmented/Aperio_FDA/'\n",
    "#BASE PATH for working from home:\n",
    "#BASE_PATH = '/home/OSEL/Desktop/HER2_data_categorical/'\n",
    "#epochs = 10\n",
    "batch_size = 32\n",
    "num_classes = 3\n",
    "#epochs = 35"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get Data - Practice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This is the version from Ravi's code:\n",
    "\n",
    "#FDA\n",
    "#X_FDA = []\n",
    "#idx_FDA = []\n",
    "#for index, image_filename in list(enumerate(BASE_PATH)):\n",
    "#\timg_file = cv2.imread(BASE_PATH + '/' + image_filename)\n",
    "#\tif img_file is not None:\n",
    "\t\t#img_file = smisc.imresize(arr = img_file, size = (600,760,3))\n",
    "#\t\timg_file = smisc.imresize(arr = img_file, size = (120,160,3))\t\t\n",
    "#\t\timg_arr = np.asarray(img_file)\n",
    "#\t\tX_FDA.append(img_arr)\n",
    "#\t\tidx_FDA.append(index)\n",
    "\n",
    "#X_FDA = np.asarray(X_FDA)\n",
    "#idx_FDA = np.asarray(idx_FDA)\n",
    "#random.seed(rs)\n",
    "#random_id = random.sample(idx_FDA, len(idx_FDA)/2)\n",
    "#random_FDA = []\n",
    "#for i in random_id:\n",
    "#\trandom_FDA.append(X_FDA[i])\n",
    "\n",
    "#random_FDA = np.asarray(random_FDA)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get Data - Real"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data(folder):\n",
    "    X = []\n",
    "    y = []\n",
    "    filenames = []\n",
    "\n",
    "    for hclass in os.listdir(folder):\n",
    "        if not hclass.startswith('.'):\n",
    "            if hclass in [\"1\"]:\n",
    "                label = 1\n",
    "            else: #label must be 1 or 2\n",
    "                if hclass in [\"2\"]:\n",
    "                    label = 2\n",
    "                else:\n",
    "                    label = 3\n",
    "            for image_filename in os.listdir(folder + hclass):\n",
    "                filename = folder + hclass + '/' + image_filename\n",
    "                img_file = cv2.imread(folder + hclass + '/' + image_filename)\n",
    "                \n",
    "                if img_file is not None:\n",
    "                    img_file = scipy.misc.imresize(arr=img_file, size=(120, 160, 3))\n",
    "                    img_arr = np.asarray(img_file)\n",
    "                    X.append(img_arr)\n",
    "                    y.append(label)\n",
    "                    filenames.append(filename)\n",
    "\n",
    "    X = np.asarray(X)\n",
    "    y = np.asarray(y)\n",
    "    z = np.asarray(filenames)\n",
    "    return X,y,z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:20: DeprecationWarning: `imresize` is deprecated!\n",
      "`imresize` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``skimage.transform.resize`` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "305\n",
      "305\n",
      "305\n",
      "[[ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]]\n"
     ]
    }
   ],
   "source": [
    "X, y, z = get_data(BASE_PATH)\n",
    "\n",
    "\n",
    "#print(X)\n",
    "#print(y)\n",
    "#print(z)\n",
    "print(len(X))\n",
    "print(len(y))\n",
    "print(len(z))\n",
    "\n",
    "#INTEGER ENCODE\n",
    "#https://machinelearningmastery.com/how-to-one-hot-encode-sequence-data-in-python/\n",
    "encoder = LabelEncoder()\n",
    "y_cat = np_utils.to_categorical(encoder.fit_transform(y))\n",
    "print(y_cat)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Old Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#encoder = LabelEncoder()\n",
    "#encoder.fit(y)\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=10)\n",
    "\n",
    "#encoded_y_train = encoder.transform(y_train)\n",
    "#encoded_y_test = encoder.transform(y_test)\n",
    "\n",
    "#y_train = np_utils.to_categorical(encoded_y_train)\n",
    "#y_test = np_utils.to_categorical(encoded_y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit Model with K-Fold X-Val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "305\n",
      "KFold(n_splits=305, random_state=5, shuffle=True)\n"
     ]
    }
   ],
   "source": [
    "kf = KFold(n_splits = 305, random_state=5, shuffle=True)\n",
    "print(kf.get_n_splits(y))\n",
    "print(kf)\n",
    "\n",
    "\n",
    "#for train_index, test_index in kf.split(y):\n",
    "#    X_train, X_test = X[train_index], X[test_index]\n",
    "#    print(train_index, test_index)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold #1\n",
      "Epoch 00081: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  5.17372209e-05   9.99897122e-01   5.10833306e-05]]\n",
      "[1]\n",
      "fold #2\n",
      "Epoch 00027: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.00209497  0.41113618  0.58676881]]\n",
      "[2]\n",
      "fold #3\n",
      "Epoch 00044: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  6.90236388e-13   9.98480141e-01   1.51989690e-03]]\n",
      "[1]\n",
      "fold #4\n",
      "Epoch 00076: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[ 0.12862791  0.86760151  0.00377062]]\n",
      "[1]\n",
      "fold #5\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   3.75530870e-21   2.54284024e-21]]\n",
      "[0]\n",
      "fold #6\n",
      "Epoch 00035: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  8.64286448e-21   3.91668655e-05   9.99960780e-01]]\n",
      "[2]\n",
      "fold #7\n",
      "Epoch 00039: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.69804781e-09   1.62188277e-01   8.37811708e-01]]\n",
      "[2]\n",
      "fold #8\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.79802580e-22   4.57583304e-25]]\n",
      "[0]\n",
      "fold #9\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.56588421e-25   3.65642790e-21]]\n",
      "[0]\n",
      "fold #10\n",
      "Epoch 00026: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[ 0.27439344  0.71668291  0.00892364]]\n",
      "[1]\n",
      "fold #11\n",
      "Epoch 00031: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   4.71665689e-14   2.67071779e-19]]\n",
      "[0]\n",
      "fold #12\n",
      "Epoch 00033: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.90786373e-08   3.85734320e-01   6.14265621e-01]]\n",
      "[2]\n",
      "fold #13\n",
      "Epoch 00108: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  4.53772930e-09   9.58505392e-01   4.14946340e-02]]\n",
      "[1]\n",
      "fold #14\n",
      "Epoch 00059: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.03451203  0.95964903  0.00583889]]\n",
      "[1]\n",
      "fold #15\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   2.15553664e-15   2.31617337e-22]]\n",
      "[0]\n",
      "fold #16\n",
      "Epoch 00090: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  6.08027573e-10   6.14030007e-03   9.93859649e-01]]\n",
      "[2]\n",
      "fold #17\n",
      "Epoch 00032: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  0.00000000e+00   3.35368804e-14   1.00000000e+00]]\n",
      "[2]\n",
      "fold #18\n",
      "Epoch 00031: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.41242189e-28   9.04814533e-06   9.99990940e-01]]\n",
      "[2]\n",
      "fold #19\n",
      "Epoch 00033: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[ 0.07256068  0.92648673  0.00095257]]\n",
      "[1]\n",
      "fold #20\n",
      "Epoch 00048: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  9.31726674e-07   9.62188244e-01   3.78108323e-02]]\n",
      "[1]\n",
      "fold #21\n",
      "Epoch 00096: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  7.84447999e-04   9.99206960e-01   8.63537662e-06]]\n",
      "[1]\n",
      "fold #22\n",
      "Epoch 00040: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   6.88277824e-10   1.96672512e-13]]\n",
      "[0]\n",
      "fold #23\n",
      "Epoch 00046: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.37252185e-14   1.03196166e-02   9.89680350e-01]]\n",
      "[2]\n",
      "fold #24\n",
      "Epoch 00086: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  8.31664947e-05   9.99916792e-01   5.41122311e-08]]\n",
      "[1]\n",
      "fold #25\n",
      "Epoch 00087: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.97877011e-06   8.99685204e-01   1.00312926e-01]]\n",
      "[1]\n",
      "fold #26\n",
      "Epoch 00063: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  6.15568191e-04   8.59578013e-01   1.39806464e-01]]\n",
      "[1]\n",
      "fold #27\n",
      "Epoch 00039: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  3.45734067e-17   5.34495106e-04   9.99465525e-01]]\n",
      "[2]\n",
      "fold #28\n",
      "Epoch 00031: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   3.57397358e-22   1.13635556e-31]]\n",
      "[0]\n",
      "fold #29\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.00076602  0.67234302  0.32689101]]\n",
      "[1]\n",
      "fold #30\n",
      "Epoch 00101: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  3.11080344e-13   1.00000000e+00   2.75021836e-08]]\n",
      "[1]\n",
      "fold #31\n",
      "Epoch 00083: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  9.28334339e-05   9.99762118e-01   1.45101498e-04]]\n",
      "[1]\n",
      "fold #32\n",
      "Epoch 00110: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  6.24413371e-01   3.75227630e-01   3.58997931e-04]]\n",
      "[0]\n",
      "fold #33\n",
      "Epoch 00069: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.61648597e-04   9.99763191e-01   7.52391352e-05]]\n",
      "[1]\n",
      "fold #34\n",
      "Epoch 00082: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  2.64875343e-05   9.99234676e-01   7.38860166e-04]]\n",
      "[1]\n",
      "fold #35\n",
      "Epoch 00094: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  3.52527138e-07   9.99997973e-01   1.63148115e-06]]\n",
      "[1]\n",
      "fold #36\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  2.04444700e-20   1.79809323e-08   1.00000000e+00]]\n",
      "[2]\n",
      "fold #37\n",
      "Epoch 00032: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  0.00000000e+00   6.68044562e-16   1.00000000e+00]]\n",
      "[2]\n",
      "fold #38\n",
      "Epoch 00095: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  2.84606595e-05   9.99917865e-01   5.35881372e-05]]\n",
      "[1]\n",
      "fold #39\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   5.76539994e-27   1.51445110e-25]]\n",
      "[0]\n",
      "fold #40\n",
      "Epoch 00031: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.22280334  0.76714361  0.01005309]]\n",
      "[1]\n",
      "fold #41\n",
      "Epoch 00059: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.38001198  0.61509532  0.00489269]]\n",
      "[1]\n",
      "fold #42\n",
      "Epoch 00117: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  2.90250336e-03   9.97097015e-01   4.71797620e-07]]\n",
      "[1]\n",
      "fold #43\n",
      "Epoch 00045: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.04815359  0.95064765  0.00119872]]\n",
      "[1]\n",
      "fold #44\n",
      "Epoch 00040: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  5.21576736e-25   7.24805577e-05   9.99927521e-01]]\n",
      "[2]\n",
      "fold #45\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   7.85277189e-24   1.70847299e-35]]\n",
      "[0]\n",
      "fold #46\n",
      "Epoch 00033: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  8.70396681e-25   9.57197537e-13   1.00000000e+00]]\n",
      "[2]\n",
      "fold #47\n",
      "Epoch 00043: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  5.23349892e-27   3.85702208e-07   9.99999642e-01]]\n",
      "[2]\n",
      "fold #48\n",
      "Epoch 00072: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.11527891  0.87376034  0.01096069]]\n",
      "[1]\n",
      "fold #49\n",
      "Epoch 00098: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  6.17665146e-03   9.93491948e-01   3.31433752e-04]]\n",
      "[1]\n",
      "fold #50\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  7.07794630e-12   1.06500639e-02   9.89349902e-01]]\n",
      "[2]\n",
      "fold #51\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  2.91934120e-04   4.50754821e-01   5.48953235e-01]]\n",
      "[2]\n",
      "fold #52\n",
      "Epoch 00098: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.69266469e-03   9.98307109e-01   2.68004300e-07]]\n",
      "[1]\n",
      "fold #53\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   6.06542705e-22   0.00000000e+00]]\n",
      "[0]\n",
      "fold #54\n",
      "Epoch 00034: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.29958679e-13   3.99181619e-02   9.60081875e-01]]\n",
      "[2]\n",
      "fold #55\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.64204120e-17   9.28560899e-38]]\n",
      "[0]\n",
      "fold #56\n",
      "Epoch 00069: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.07729911  0.9083873   0.0143135 ]]\n",
      "[1]\n",
      "fold #57\n",
      "Epoch 00048: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  5.47842993e-10   6.46808863e-01   3.53191108e-01]]\n",
      "[1]\n",
      "fold #58\n",
      "Epoch 00083: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  3.07227038e-02   9.69161987e-01   1.15288560e-04]]\n",
      "[1]\n",
      "fold #59\n",
      "Epoch 00026: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[ 0.25347418  0.68925053  0.05727535]]\n",
      "[1]\n",
      "fold #60\n",
      "Epoch 00026: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[ 0.27230385  0.71516043  0.01253569]]\n",
      "[1]\n",
      "fold #61\n",
      "Epoch 00031: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  3.60152073e-04   1.90371439e-01   8.09268415e-01]]\n",
      "[2]\n",
      "fold #62\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00050: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.37248094e-15   2.21357662e-02   9.77864206e-01]]\n",
      "[2]\n",
      "fold #63\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.66618741e-09   1.10970922e-02   9.88902867e-01]]\n",
      "[2]\n",
      "fold #64\n",
      "Epoch 00033: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.15390849  0.81993818  0.02615336]]\n",
      "[1]\n",
      "fold #65\n",
      "Epoch 00105: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.51564673e-05   9.99419332e-01   5.65535505e-04]]\n",
      "[1]\n",
      "fold #66\n",
      "Epoch 00035: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.00576043  0.7147249   0.2795147 ]]\n",
      "[1]\n",
      "fold #67\n",
      "Epoch 00031: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[ 0.3013908   0.65233558  0.04627356]]\n",
      "[1]\n",
      "fold #68\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   7.94764109e-24   7.06472214e-20]]\n",
      "[0]\n",
      "fold #69\n",
      "Epoch 00072: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  3.01041506e-14   5.27039349e-01   4.72960621e-01]]\n",
      "[1]\n",
      "fold #70\n",
      "Epoch 00044: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.21245626  0.77235401  0.01518976]]\n",
      "[1]\n",
      "fold #71\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   8.94579372e-18   2.57574682e-24]]\n",
      "[0]\n",
      "fold #72\n",
      "Epoch 00073: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.20713842  0.78889543  0.00396615]]\n",
      "[1]\n",
      "fold #73\n",
      "Epoch 00042: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.38943353  0.60946608  0.0011004 ]]\n",
      "[1]\n",
      "fold #74\n",
      "Epoch 00037: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[ 0.0030251   0.82385665  0.17311825]]\n",
      "[1]\n",
      "fold #75\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   9.21554397e-27   6.90065938e-24]]\n",
      "[0]\n",
      "fold #76\n",
      "Epoch 00076: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  2.79520033e-03   9.96964633e-01   2.40207155e-04]]\n",
      "[1]\n",
      "fold #77\n",
      "Epoch 00035: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.17402518e-10   1.00007288e-01   8.99992704e-01]]\n",
      "[2]\n",
      "fold #78\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[ 0.00340288  0.94749576  0.04910139]]\n",
      "[1]\n",
      "fold #79\n",
      "Epoch 00114: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  2.06279248e-01   7.93211102e-01   5.09637292e-04]]\n",
      "[1]\n",
      "fold #80\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   5.11272844e-22   4.22648254e-26]]\n",
      "[0]\n",
      "fold #81\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.90681347e-26   7.32888696e-33]]\n",
      "[0]\n",
      "fold #82\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   8.65613607e-29   1.03682523e-30]]\n",
      "[0]\n",
      "fold #83\n",
      "Epoch 00049: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.05690732  0.94057202  0.0025207 ]]\n",
      "[1]\n",
      "fold #84\n",
      "Epoch 00095: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  2.26227823e-03   9.97468829e-01   2.68835400e-04]]\n",
      "[1]\n",
      "fold #85\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.73262673e-19   7.35486827e-09   1.00000000e+00]]\n",
      "[2]\n",
      "fold #86\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   9.17958747e-33   2.89902575e-34]]\n",
      "[0]\n",
      "fold #87\n",
      "Epoch 00097: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  6.10224068e-01   3.89768422e-01   7.47589002e-06]]\n",
      "[0]\n",
      "fold #88\n",
      "Epoch 00075: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.28617617  0.70736271  0.00646113]]\n",
      "[1]\n",
      "fold #89\n",
      "Epoch 00033: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.10130135  0.88860953  0.01008909]]\n",
      "[1]\n",
      "fold #90\n",
      "Epoch 00076: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  3.25558288e-03   9.96702015e-01   4.24566897e-05]]\n",
      "[1]\n",
      "fold #91\n",
      "Epoch 00104: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  8.51455679e-14   9.99938726e-01   6.12206350e-05]]\n",
      "[1]\n",
      "fold #92\n",
      "Epoch 00033: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   2.37758927e-11   9.12772416e-20]]\n",
      "[0]\n",
      "fold #93\n",
      "Epoch 00042: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  2.23676255e-16   1.99204966e-04   9.99800861e-01]]\n",
      "[2]\n",
      "fold #94\n",
      "Epoch 00095: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  2.03325115e-02   9.79620159e-01   4.72698375e-05]]\n",
      "[1]\n",
      "fold #95\n",
      "Epoch 00079: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  3.19416082e-04   9.99646306e-01   3.43545180e-05]]\n",
      "[1]\n",
      "fold #96\n",
      "Epoch 00035: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.08409513e-04   5.04527271e-01   4.95364368e-01]]\n",
      "[1]\n",
      "fold #97\n",
      "Epoch 00081: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  9.82039392e-01   1.79588944e-02   1.72154307e-06]]\n",
      "[0]\n",
      "fold #98\n",
      "Epoch 00104: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.33419226e-05   9.98839557e-01   1.14709383e-03]]\n",
      "[1]\n",
      "fold #99\n",
      "Epoch 00035: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.25767973  0.72515643  0.01716389]]\n",
      "[1]\n",
      "fold #100\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   4.61708501e-20   3.31515979e-29]]\n",
      "[0]\n",
      "fold #101\n",
      "Epoch 00095: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.77591289e-06   9.99374330e-01   6.23878499e-04]]\n",
      "[1]\n",
      "fold #102\n",
      "Epoch 00036: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[ 0.12958986  0.86405927  0.00635093]]\n",
      "[1]\n",
      "fold #103\n",
      "Epoch 00074: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  2.36466349e-18   1.64323428e-04   9.99835610e-01]]\n",
      "[2]\n",
      "fold #104\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   3.65707302e-19   6.01655312e-21]]\n",
      "[0]\n",
      "fold #105\n",
      "Epoch 00049: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.29748186  0.68445557  0.0180626 ]]\n",
      "[1]\n",
      "fold #106\n",
      "Epoch 00026: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[ 0.13679267  0.86014456  0.00306274]]\n",
      "[1]\n",
      "fold #107\n",
      "Epoch 00106: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.16401250e-04   9.99881387e-01   2.14363104e-06]]\n",
      "[1]\n",
      "fold #108\n",
      "Epoch 00034: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  7.30437456e-18   1.24051412e-05   9.99987602e-01]]\n",
      "[2]\n",
      "fold #109\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   9.89949428e-16   8.90437034e-23]]\n",
      "[0]\n",
      "fold #110\n",
      "Epoch 00044: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  8.26267238e-23   6.91563673e-06   9.99993086e-01]]\n",
      "[2]\n",
      "fold #111\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.59701176e-18   4.47717812e-30]]\n",
      "[0]\n",
      "fold #112\n",
      "Epoch 00072: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  9.05601519e-06   9.92113113e-01   7.87772983e-03]]\n",
      "[1]\n",
      "fold #113\n",
      "Epoch 00041: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  7.13032289e-29   4.90153325e-05   9.99951005e-01]]\n",
      "[2]\n",
      "fold #114\n",
      "Epoch 00039: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  3.98514052e-15   2.73847412e-02   9.72615242e-01]]\n",
      "[2]\n",
      "fold #115\n",
      "Epoch 00085: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.13825275  0.8343631   0.02738423]]\n",
      "[1]\n",
      "fold #116\n",
      "Epoch 00050: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.23983957  0.75915563  0.00100485]]\n",
      "[1]\n",
      "fold #117\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.50394321e-19   2.96126119e-26]]\n",
      "[0]\n",
      "fold #118\n",
      "Epoch 00094: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.91389245e-06   9.99566019e-01   4.32023953e-04]]\n",
      "[1]\n",
      "fold #119\n",
      "Epoch 00055: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  3.20475525e-12   9.99999523e-01   5.05593789e-07]]\n",
      "[1]\n",
      "fold #120\n",
      "Epoch 00080: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  8.59025342e-04   9.99118507e-01   2.24782671e-05]]\n",
      "[1]\n",
      "fold #121\n",
      "Epoch 00066: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  2.11288338e-04   9.84685302e-01   1.51034240e-02]]\n",
      "[1]\n",
      "fold #122\n",
      "Epoch 00032: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  8.87638471e-32   8.92899144e-13   1.00000000e+00]]\n",
      "[2]\n",
      "fold #123\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00046: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  7.07171992e-08   2.90037505e-02   9.70996201e-01]]\n",
      "[2]\n",
      "fold #124\n",
      "Epoch 00061: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.39890808e-05   9.99596894e-01   3.89165420e-04]]\n",
      "[1]\n",
      "fold #125\n",
      "Epoch 00080: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  7.70834321e-03   9.92288768e-01   2.83593408e-06]]\n",
      "[1]\n",
      "fold #126\n",
      "Epoch 00032: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  2.02966916e-33   1.83569815e-09   1.00000000e+00]]\n",
      "[2]\n",
      "fold #127\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.17382048e-21   1.37802801e-27]]\n",
      "[0]\n",
      "fold #128\n",
      "Epoch 00026: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.04826087  0.84653705  0.10520215]]\n",
      "[1]\n",
      "fold #129\n",
      "Epoch 00079: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.49403850e-03   9.98505712e-01   2.84101390e-07]]\n",
      "[1]\n",
      "fold #130\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   4.11092245e-21   9.24738252e-26]]\n",
      "[0]\n",
      "fold #131\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   3.09297242e-20   1.78440133e-29]]\n",
      "[0]\n",
      "fold #132\n",
      "Epoch 00027: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.02310516e-02   9.89159048e-01   6.09914248e-04]]\n",
      "[1]\n",
      "fold #133\n",
      "Epoch 00034: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  5.37785659e-21   7.71970932e-09   1.00000000e+00]]\n",
      "[2]\n",
      "fold #134\n",
      "Epoch 00049: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  3.02617476e-16   5.13671160e-01   4.86328781e-01]]\n",
      "[1]\n",
      "fold #135\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  9.49106853e-22   3.34613615e-10   1.00000000e+00]]\n",
      "[2]\n",
      "fold #136\n",
      "Epoch 00026: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[ 0.21390472  0.76274729  0.02334794]]\n",
      "[1]\n",
      "fold #137\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   2.88980940e-17   1.80434703e-22]]\n",
      "[0]\n",
      "fold #138\n",
      "Epoch 00031: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.69000664e-12   4.68927348e-04   9.99531150e-01]]\n",
      "[2]\n",
      "fold #139\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  8.38048086e-20   2.27994157e-08   1.00000000e+00]]\n",
      "[2]\n",
      "fold #140\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.12531195e-24   1.24695605e-33]]\n",
      "[0]\n",
      "fold #141\n",
      "Epoch 00101: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  5.28970219e-08   9.90216017e-01   9.78394970e-03]]\n",
      "[1]\n",
      "fold #142\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.55816053e-18   6.39968334e-07   9.99999404e-01]]\n",
      "[2]\n",
      "fold #143\n",
      "Epoch 00075: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  4.41956757e-07   9.99768794e-01   2.30714155e-04]]\n",
      "[1]\n",
      "fold #144\n",
      "Epoch 00097: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  3.05930786e-02   9.69395876e-01   1.11339032e-05]]\n",
      "[1]\n",
      "fold #145\n",
      "Epoch 00065: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.3301546   0.66901696  0.00082846]]\n",
      "[1]\n",
      "fold #146\n",
      "Epoch 00098: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[ 0.65549994  0.34271643  0.00178366]]\n",
      "[0]\n",
      "fold #147\n",
      "Epoch 00032: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  4.56340690e-24   5.86388712e-14   1.00000000e+00]]\n",
      "[2]\n",
      "fold #148\n",
      "Epoch 00039: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  4.59392131e-16   6.40688988e-04   9.99359310e-01]]\n",
      "[2]\n",
      "fold #149\n",
      "Epoch 00083: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  2.10234735e-04   9.84191537e-01   1.55982096e-02]]\n",
      "[1]\n",
      "fold #150\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   4.31429342e-29   3.73949060e-25]]\n",
      "[0]\n",
      "fold #151\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   8.14212774e-34   9.03739037e-37]]\n",
      "[0]\n",
      "fold #152\n",
      "Epoch 00081: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.40074907e-09   9.99497056e-01   5.02946379e-04]]\n",
      "[1]\n",
      "fold #153\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   4.14374447e-16   6.41841453e-26]]\n",
      "[0]\n",
      "fold #154\n",
      "Epoch 00054: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.00167035  0.99540663  0.00292303]]\n",
      "[1]\n",
      "fold #155\n",
      "Epoch 00034: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   2.45377230e-08   2.57033327e-13]]\n",
      "[0]\n",
      "fold #156\n",
      "Epoch 00088: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  3.87770588e-12   9.99998569e-01   1.44292710e-06]]\n",
      "[1]\n",
      "fold #157\n",
      "Epoch 00088: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  2.70209342e-01   7.29539454e-01   2.51181889e-04]]\n",
      "[1]\n",
      "fold #158\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   3.13821388e-30   2.44838640e-27]]\n",
      "[0]\n",
      "fold #159\n",
      "Epoch 00043: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  2.61450553e-24   2.90887101e-06   9.99997139e-01]]\n",
      "[2]\n",
      "fold #160\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.12435185  0.84348094  0.03216723]]\n",
      "[1]\n",
      "fold #161\n",
      "Epoch 00032: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  2.57025161e-23   7.99546193e-12   1.00000000e+00]]\n",
      "[2]\n",
      "fold #162\n",
      "Epoch 00074: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.43839759e-04   9.99855638e-01   4.27872948e-07]]\n",
      "[1]\n",
      "fold #163\n",
      "Epoch 00092: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  5.60564315e-03   9.94392872e-01   1.44237606e-06]]\n",
      "[1]\n",
      "fold #164\n",
      "Epoch 00086: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.00981449  0.98591959  0.00426594]]\n",
      "[1]\n",
      "fold #165\n",
      "Epoch 00049: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  3.75188283e-11   7.59186864e-01   2.40813136e-01]]\n",
      "[1]\n",
      "fold #166\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.03077069e-17   4.13903878e-15]]\n",
      "[0]\n",
      "fold #167\n",
      "Epoch 00043: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.04514910e-12   7.72534477e-05   9.99922752e-01]]\n",
      "[2]\n",
      "fold #168\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.83052459e-23   7.75091955e-34]]\n",
      "[0]\n",
      "fold #169\n",
      "Epoch 00084: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  4.60437986e-05   9.97099996e-01   2.85406224e-03]]\n",
      "[1]\n",
      "fold #170\n",
      "Epoch 00032: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  7.03955116e-25   1.37325280e-08   1.00000000e+00]]\n",
      "[2]\n",
      "fold #171\n",
      "Epoch 00084: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  2.95772708e-07   7.06748366e-01   2.93251276e-01]]\n",
      "[1]\n",
      "fold #172\n",
      "Epoch 00083: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.53881079  0.4551827   0.00600647]]\n",
      "[0]\n",
      "fold #173\n",
      "Epoch 00036: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.29882112  0.69271952  0.00845937]]\n",
      "[1]\n",
      "fold #174\n",
      "Epoch 00046: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[ 0.25454268  0.74329799  0.00215932]]\n",
      "[1]\n",
      "fold #175\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   2.74868650e-16   4.31478201e-24]]\n",
      "[0]\n",
      "fold #176\n",
      "Epoch 00044: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.00270803  0.96691161  0.03038035]]\n",
      "[1]\n",
      "fold #177\n",
      "Epoch 00072: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  3.87208196e-13   6.12018764e-01   3.87981296e-01]]\n",
      "[1]\n",
      "fold #178\n",
      "Epoch 00145: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  9.90371555e-02   9.00655985e-01   3.06865695e-04]]\n",
      "[1]\n",
      "fold #179\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   6.08709888e-11   4.80333240e-21]]\n",
      "[0]\n",
      "fold #180\n",
      "Epoch 00044: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.1525338   0.84625691  0.00120929]]\n",
      "[1]\n",
      "fold #181\n",
      "Epoch 00103: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  2.40056079e-06   9.99995828e-01   1.76260164e-06]]\n",
      "[1]\n",
      "fold #182\n",
      "Epoch 00033: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.64963140e-10   2.23397350e-18]]\n",
      "[0]\n",
      "fold #183\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  3.30223023e-12   9.79626420e-05   9.99902010e-01]]\n",
      "[2]\n",
      "fold #184\n",
      "Epoch 00070: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.99371379e-14   9.66390431e-01   3.36095728e-02]]\n",
      "[1]\n",
      "fold #185\n",
      "Epoch 00119: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  9.34360269e-03   9.90355492e-01   3.00970074e-04]]\n",
      "[1]\n",
      "fold #186\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.21883719e-18   2.53369200e-27]]\n",
      "[0]\n",
      "fold #187\n",
      "Epoch 00085: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.53645361e-19   6.27342843e-06   9.99993682e-01]]\n",
      "[2]\n",
      "fold #188\n",
      "Epoch 00065: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  2.22921237e-10   9.09491956e-01   9.05080438e-02]]\n",
      "[1]\n",
      "fold #189\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   4.56279751e-24   1.75494167e-29]]\n",
      "[0]\n",
      "fold #190\n",
      "Epoch 00032: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  2.71487661e-29   6.51782148e-06   9.99993443e-01]]\n",
      "[2]\n",
      "fold #191\n",
      "Epoch 00076: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  4.15767193e-01   5.84147871e-01   8.49549178e-05]]\n",
      "[1]\n",
      "fold #192\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   2.50627104e-31   7.68574000e-32]]\n",
      "[0]\n",
      "fold #193\n",
      "Epoch 00099: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  7.13762859e-07   9.99998689e-01   5.40360304e-07]]\n",
      "[1]\n",
      "fold #194\n",
      "Epoch 00074: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[ 0.39645809  0.60157925  0.0019627 ]]\n",
      "[1]\n",
      "fold #195\n",
      "Epoch 00080: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  3.72006238e-04   9.99616504e-01   1.14000013e-05]]\n",
      "[1]\n",
      "fold #196\n",
      "Epoch 00098: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  4.82226005e-06   9.98381734e-01   1.61341822e-03]]\n",
      "[1]\n",
      "fold #197\n",
      "Epoch 00099: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  5.55607316e-04   9.99444425e-01   4.00101499e-08]]\n",
      "[1]\n",
      "fold #198\n",
      "Epoch 00074: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.00150802  0.9919064   0.00658558]]\n",
      "[1]\n",
      "fold #199\n",
      "Epoch 00047: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.25689992  0.73086333  0.01223679]]\n",
      "[1]\n",
      "fold #200\n",
      "Epoch 00091: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  3.52768240e-07   9.98821676e-01   1.17790385e-03]]\n",
      "[1]\n",
      "fold #201\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.72303580e-20   8.20027411e-23]]\n",
      "[0]\n",
      "fold #202\n",
      "Epoch 00138: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  2.98667146e-06   9.99996781e-01   2.23036409e-07]]\n",
      "[1]\n",
      "fold #203\n",
      "Epoch 00046: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  6.31150560e-16   2.74650473e-02   9.72534955e-01]]\n",
      "[2]\n",
      "fold #204\n",
      "Epoch 00103: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  3.33957956e-04   9.91435945e-01   8.23009759e-03]]\n",
      "[1]\n",
      "fold #205\n",
      "Epoch 00038: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  3.14945349e-14   5.41428162e-04   9.99458611e-01]]\n",
      "[2]\n",
      "fold #206\n",
      "Epoch 00031: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.17399633  0.80046034  0.02554328]]\n",
      "[1]\n",
      "fold #207\n",
      "Epoch 00057: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  5.33843995e-04   9.69055891e-01   3.04102562e-02]]\n",
      "[1]\n",
      "fold #208\n",
      "Epoch 00039: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.30557051  0.68300962  0.01141991]]\n",
      "[1]\n",
      "fold #209\n",
      "Epoch 00083: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  3.28578899e-04   9.99259531e-01   4.11878398e-04]]\n",
      "[1]\n",
      "fold #210\n",
      "Epoch 00031: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  9.68864858e-21   6.60038246e-10   1.00000000e+00]]\n",
      "[2]\n",
      "fold #211\n",
      "Epoch 00105: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  5.97311839e-07   9.97364938e-01   2.63448851e-03]]\n",
      "[1]\n",
      "fold #212\n",
      "Epoch 00043: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.20464268  0.7899406   0.00541676]]\n",
      "[1]\n",
      "fold #213\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   6.54703512e-24   1.46179491e-23]]\n",
      "[0]\n",
      "fold #214\n",
      "Epoch 00031: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  3.60118924e-32   3.40626859e-13   1.00000000e+00]]\n",
      "[2]\n",
      "fold #215\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.18191549e-24   2.47971175e-37]]\n",
      "[0]\n",
      "fold #216\n",
      "Epoch 00073: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  5.56755753e-04   9.97605681e-01   1.83753145e-03]]\n",
      "[1]\n",
      "fold #217\n",
      "Epoch 00114: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.44955486e-01   8.55043769e-01   6.84226677e-07]]\n",
      "[1]\n",
      "fold #218\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  3.77055141e-04   8.72804224e-01   1.26818672e-01]]\n",
      "[1]\n",
      "fold #219\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.23793039e-13   1.59339200e-32]]\n",
      "[0]\n",
      "fold #220\n",
      "Epoch 00027: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   2.48994392e-26   0.00000000e+00]]\n",
      "[0]\n",
      "fold #221\n",
      "Epoch 00026: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[ 0.19040157  0.76810545  0.04149304]]\n",
      "[1]\n",
      "fold #222\n",
      "Epoch 00061: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  2.18202545e-13   3.90825897e-01   6.09174132e-01]]\n",
      "[2]\n",
      "fold #223\n",
      "Epoch 00049: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  4.24137397e-05   3.12564284e-01   6.87393308e-01]]\n",
      "[2]\n",
      "fold #224\n",
      "Epoch 00026: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[ 0.08702211  0.88413233  0.0288455 ]]\n",
      "[1]\n",
      "fold #225\n",
      "Epoch 00031: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   3.11256348e-26   3.48538644e-34]]\n",
      "[0]\n",
      "fold #226\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   7.44662894e-18   8.75823227e-23]]\n",
      "[0]\n",
      "fold #227\n",
      "Epoch 00091: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  2.35121315e-05   9.99851465e-01   1.25007762e-04]]\n",
      "[1]\n",
      "fold #228\n",
      "Epoch 00034: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  9.99999881e-01   1.67687929e-07   2.01461314e-09]]\n",
      "[0]\n",
      "fold #229\n",
      "Epoch 00031: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.22269157e-26   9.70574206e-07   9.99999046e-01]]\n",
      "[2]\n",
      "fold #230\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   5.10901624e-25   9.01526382e-34]]\n",
      "[0]\n",
      "fold #231\n",
      "Epoch 00071: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.97557820e-05   9.99357641e-01   6.22596359e-04]]\n",
      "[1]\n",
      "fold #232\n",
      "Epoch 00108: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  5.28733886e-04   9.99467552e-01   3.66357813e-06]]\n",
      "[1]\n",
      "fold #233\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   8.18501121e-26   5.00016981e-27]]\n",
      "[0]\n",
      "fold #234\n",
      "Epoch 00039: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.14431708e-29   6.75046463e-10   1.00000000e+00]]\n",
      "[2]\n",
      "fold #235\n",
      "Epoch 00033: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  3.03016793e-18   1.23415830e-07   9.99999881e-01]]\n",
      "[2]\n",
      "fold #236\n",
      "Epoch 00056: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  9.12335873e-01   8.75329301e-02   1.31157911e-04]]\n",
      "[0]\n",
      "fold #237\n",
      "Epoch 00039: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.07662632e-23   8.35559888e-07   9.99999166e-01]]\n",
      "[2]\n",
      "fold #238\n",
      "Epoch 00038: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.2112498   0.77555907  0.01319109]]\n",
      "[1]\n",
      "fold #239\n",
      "Epoch 00090: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.53962028e-05   9.99983430e-01   1.21150731e-06]]\n",
      "[1]\n",
      "fold #240\n",
      "Epoch 00110: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  9.86082613e-01   1.39135281e-02   3.84769373e-06]]\n",
      "[0]\n",
      "fold #241\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.19055468e-15   0.00000000e+00]]\n",
      "[0]\n",
      "fold #242\n",
      "Epoch 00035: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   3.33371331e-14   1.24549954e-21]]\n",
      "[0]\n",
      "fold #243\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   6.86969264e-24   7.93995088e-30]]\n",
      "[0]\n",
      "fold #244\n",
      "Epoch 00035: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  3.06171029e-19   7.99275426e-07   9.99999166e-01]]\n",
      "[2]\n",
      "fold #245\n",
      "Epoch 00095: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  2.90834423e-09   9.40093994e-01   5.99060617e-02]]\n",
      "[1]\n",
      "fold #246\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  7.70288056e-16   5.83443523e-07   9.99999404e-01]]\n",
      "[2]\n",
      "fold #247\n",
      "Epoch 00062: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.10446090e-04   9.97525394e-01   2.36416166e-03]]\n",
      "[1]\n",
      "fold #248\n",
      "Epoch 00039: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  5.69690643e-15   3.25360634e-05   9.99967456e-01]]\n",
      "[2]\n",
      "fold #249\n",
      "Epoch 00091: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  4.34957947e-05   9.99916434e-01   4.00487297e-05]]\n",
      "[1]\n",
      "fold #250\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.04992288e-24   4.00518880e-33]]\n",
      "[0]\n",
      "fold #251\n",
      "Epoch 00060: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.00385302  0.98935002  0.00679695]]\n",
      "[1]\n",
      "fold #252\n",
      "Epoch 00072: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  3.33931155e-07   9.99999642e-01   4.48172877e-08]]\n",
      "[1]\n",
      "fold #253\n",
      "Epoch 00051: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.17039528  0.82279432  0.00681031]]\n",
      "[1]\n",
      "fold #254\n",
      "Epoch 00046: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.0716368   0.92725569  0.00110752]]\n",
      "[1]\n",
      "fold #255\n",
      "Epoch 00035: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  3.52733671e-24   5.28106281e-15   1.00000000e+00]]\n",
      "[2]\n",
      "fold #256\n",
      "Epoch 00110: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.17055424e-04   9.99882817e-01   1.28144578e-07]]\n",
      "[1]\n",
      "fold #257\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.09069867  0.88962901  0.01967238]]\n",
      "[1]\n",
      "fold #258\n",
      "Epoch 00080: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  3.10268393e-03   9.96429384e-01   4.67998500e-04]]\n",
      "[1]\n",
      "fold #259\n",
      "Epoch 00039: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.18841456e-06   1.47179648e-01   8.52819145e-01]]\n",
      "[2]\n",
      "fold #260\n",
      "Epoch 00102: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  2.94360612e-11   8.98271501e-01   1.01728491e-01]]\n",
      "[1]\n",
      "fold #261\n",
      "Epoch 00132: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  7.54642906e-03   9.92416024e-01   3.75495729e-05]]\n",
      "[1]\n",
      "fold #262\n",
      "Epoch 00034: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.49406768e-17   1.07564699e-06   9.99998927e-01]]\n",
      "[2]\n",
      "fold #263\n",
      "Epoch 00043: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  2.90041811e-07   3.45479511e-02   9.65451777e-01]]\n",
      "[2]\n",
      "fold #264\n",
      "Epoch 00031: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[ 0.19910829  0.77572232  0.02516937]]\n",
      "[1]\n",
      "fold #265\n",
      "Epoch 00057: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.06586216e-04   8.80684674e-01   1.19208708e-01]]\n",
      "[1]\n",
      "fold #266\n",
      "Epoch 00089: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  4.08731983e-04   9.99524474e-01   6.69021974e-05]]\n",
      "[1]\n",
      "fold #267\n",
      "Epoch 00027: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   4.31606627e-24   2.87090827e-26]]\n",
      "[0]\n",
      "fold #268\n",
      "Epoch 00035: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  4.04908988e-35   2.11915346e-11   1.00000000e+00]]\n",
      "[2]\n",
      "fold #269\n",
      "Epoch 00072: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.59158651e-06   9.99944091e-01   5.43689130e-05]]\n",
      "[1]\n",
      "fold #270\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   5.16700701e-12   3.48054202e-16]]\n",
      "[0]\n",
      "fold #271\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   1.59170395e-16   2.62344866e-36]]\n",
      "[0]\n",
      "fold #272\n",
      "Epoch 00043: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.82961393e-13   3.88442841e-03   9.96115565e-01]]\n",
      "[2]\n",
      "fold #273\n",
      "Epoch 00035: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.46022761e-13   1.18023995e-03   9.98819768e-01]]\n",
      "[2]\n",
      "fold #274\n",
      "Epoch 00052: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.13175867  0.86042732  0.00781396]]\n",
      "[1]\n",
      "fold #275\n",
      "Epoch 00101: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  7.61154890e-02   9.23503399e-01   3.81110673e-04]]\n",
      "[1]\n",
      "fold #276\n",
      "Epoch 00031: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.2239036   0.76566267  0.01043373]]\n",
      "[1]\n",
      "fold #277\n",
      "Epoch 00031: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  5.85435129e-13   8.66126447e-06   9.99991298e-01]]\n",
      "[2]\n",
      "fold #278\n",
      "Epoch 00065: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.0766082   0.92056167  0.00283015]]\n",
      "[1]\n",
      "fold #279\n",
      "Epoch 00040: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  9.82495403e-14   7.86322271e-05   9.99921322e-01]]\n",
      "[2]\n",
      "fold #280\n",
      "Epoch 00093: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.27910098e-04   9.99871254e-01   8.54093969e-07]]\n",
      "[1]\n",
      "fold #281\n",
      "Epoch 00038: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.00791870e-25   1.06108000e-09   1.00000000e+00]]\n",
      "[2]\n",
      "fold #282\n",
      "Epoch 00048: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  9.99992013e-01   7.98754263e-06   2.26827712e-08]]\n",
      "[0]\n",
      "fold #283\n",
      "Epoch 00038: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[ 0.13512996  0.84416187  0.02070818]]\n",
      "[1]\n",
      "fold #284\n",
      "Epoch 00096: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  6.21836819e-03   9.93734658e-01   4.70153791e-05]]\n",
      "[1]\n",
      "fold #285\n",
      "Epoch 00112: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.26507928e-07   9.99989867e-01   9.98379801e-06]]\n",
      "[1]\n",
      "fold #286\n",
      "Epoch 00084: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  8.01855028e-02   9.19725537e-01   8.88978320e-05]]\n",
      "[1]\n",
      "fold #287\n",
      "Epoch 00060: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.196577    0.80173963  0.00168332]]\n",
      "[1]\n",
      "fold #288\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  4.35233390e-20   1.67713166e-04   9.99832273e-01]]\n",
      "[2]\n",
      "fold #289\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.29928133e-28   3.98159873e-11   1.00000000e+00]]\n",
      "[2]\n",
      "fold #290\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.12589343  0.85315883  0.02094772]]\n",
      "[1]\n",
      "fold #291\n",
      "Epoch 00098: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.34672837e-06   9.99822199e-01   1.76473433e-04]]\n",
      "[1]\n",
      "fold #292\n",
      "Epoch 00091: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  9.65985295e-04   9.97913539e-01   1.12051028e-03]]\n",
      "[1]\n",
      "fold #293\n",
      "Epoch 00033: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.33818423e-09   5.08776912e-03   9.94912207e-01]]\n",
      "[2]\n",
      "fold #294\n",
      "Epoch 00032: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  2.78377752e-06   1.73697069e-01   8.26300144e-01]]\n",
      "[2]\n",
      "fold #295\n",
      "Epoch 00091: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  5.12217980e-07   9.99983430e-01   1.61197149e-05]]\n",
      "[1]\n",
      "fold #296\n",
      "Epoch 00029: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  5.57926825e-08   2.40337756e-03   9.97596562e-01]]\n",
      "[2]\n",
      "fold #297\n",
      "Epoch 00030: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  2.35089059e-36   3.90877457e-15   1.00000000e+00]]\n",
      "[2]\n",
      "fold #298\n",
      "Epoch 00085: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.95101279e-06   9.99997616e-01   4.88406272e-07]]\n",
      "[1]\n",
      "fold #299\n",
      "Epoch 00026: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[ 0.28348634  0.65199775  0.06451585]]\n",
      "[1]\n",
      "fold #300\n",
      "Epoch 00028: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "[[  1.00000000e+00   7.17743757e-17   1.28419311e-28]]\n",
      "[0]\n",
      "fold #301\n",
      "Epoch 00118: early stopping\n",
      "Fold Score (accuracy): 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[  1.14670439e-13   3.41033846e-01   6.58966184e-01]]\n",
      "[2]\n",
      "fold #302\n",
      "Epoch 00109: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.21570569  0.65462613  0.12966813]]\n",
      "[1]\n",
      "fold #303\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00063: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "[[ 0.00925578  0.94050062  0.05024357]]\n",
      "[1]\n",
      "fold #304\n",
      "Epoch 00054: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  3.06544627e-23   8.12315477e-08   9.99999881e-01]]\n",
      "[2]\n",
      "fold #305\n",
      "Epoch 00042: early stopping\n",
      "Fold Score (accuracy): 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "[[  1.14051691e-29   1.90674036e-04   9.99809325e-01]]\n",
      "[2]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "oos_y = []\n",
    "oos_pred = []\n",
    "fold = 0\n",
    "\n",
    "\n",
    "for train, test in kf.split(y_cat):\n",
    "    fold+=1\n",
    "    print(\"fold #{}\".format(fold))\n",
    "    \n",
    "    random.shuffle(train)\n",
    "    random.shuffle(test)\n",
    "    \n",
    "    X_train = X[train]\n",
    "    y_train = y_cat[train]\n",
    "    X_test = X[test]\n",
    "    y_test = y_cat[test]\n",
    "    \n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Lambda(lambda x: x * 1./255., input_shape=(120, 160, 3), output_shape=(120, 160, 3)))\n",
    "    model.add(Conv2D(32, (3, 3), input_shape=(120, 160, 3)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "    model.add(Conv2D(32, (3, 3)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "    model.add(Conv2D(64, (3, 3)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "    model.add(Flatten())  # this converts our 3D feature maps to 1D feature vectors\n",
    "    model.add(Dense(64))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.7))\n",
    "    model.add(Dense(3))\n",
    "    model.add(Activation('softmax'))\n",
    "\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                optimizer='rmsprop',\n",
    "                metrics=['accuracy'])\n",
    "    \n",
    "    monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, patience=25, verbose=1, mode='auto')\n",
    "   \n",
    "    \n",
    "    model.fit(\n",
    "            X_train,\n",
    "            y_train,\n",
    "            validation_data=(X_test,y_test),\n",
    "            callbacks=[monitor],\n",
    "            shuffle=True,\n",
    "            batch_size=batch_size,\n",
    "            verbose=0,\n",
    "            epochs=1000)\n",
    "        \n",
    "    pred = model.predict(X_test)\n",
    "        \n",
    "    oos_y.append(y_test)\n",
    "    #pred = np.argmax(pred)\n",
    "    predx = np.argmax(pred,axis=1)\n",
    "    oos_pred.append(predx)\n",
    "    \n",
    "        #measure the fold's accuracy\n",
    "    y_compare = np.argmax(y_test,axis=1) #for accuracy calculation\n",
    "    score = metrics.accuracy_score(y_compare, predx)\n",
    "    print(\"Fold Score (accuracy): {}\".format(score))\n",
    "    \n",
    "    print(y_test)\n",
    "    #print(test)\n",
    "    print(pred)\n",
    "    print(predx)\n",
    "    #print(z[test])\n",
    "    #print(oos_y)\n",
    "    \n",
    "    \n",
    "    #outlist = []\n",
    "    \n",
    "    #outline = index[0] + '\\t' + str(z[test]) + '\\t' + str(int(pred[0])) + '\\t' + str(int(pred[1])) + '\\t' + str(int(pred[2])) + '\\t' + str(int(y[test])) + '\\t' + str(predx) + \\n'\n",
    "    #outlist.append(outline)\n",
    "    #with open(\"results.txt\", \"w\") as f:\n",
    "        #f.writelines(outlist)\n",
    "        #f.writelines(z[test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
